{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " extract the text from an HTM.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jason1416/Predicting-Movie-Reviews-with-BERT/blob/master/extract_the_text_from_an_HTM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RAi-pI8mO5ML",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "url = \"http://news.bbc.co.uk/2/hi/health/2284783.stm\"\n",
        "html = request.urlopen(url).read().decode('utf8')\n",
        "from bs4 import BeautifulSoup as bs\n",
        "soup = bs(html, 'html.parser')\n",
        "\n",
        " \n",
        "for script in soup([\"script\", \"style\"]):\n",
        "    script.extract()    # rip it out\n",
        "\n",
        "# get text\n",
        "text = soup.get_text()\n",
        "\n",
        "# break into lines and remove leading and trailing space on each\n",
        "lines = (line.strip() for line in text.splitlines())\n",
        "# break multi-headlines into a line each\n",
        "chunks = (phrase.strip() for line in lines for phrase in line.split(\"  \"))\n",
        "# drop blank lines\n",
        "text = '\\n'.join(chunk for chunk in chunks if chunk)\n",
        "\n",
        "print(text)\n",
        "#print(soup.get_text())\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zEr0FLoE604H",
        "colab_type": "text"
      },
      "source": [
        "remove remove punctuation from each word\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jOTDHqcXzxOE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import string\n",
        "print(string.punctuation)\n",
        "words = text.split()\n",
        "\n",
        "table = str.maketrans('', '', string.punctuation)\n",
        "stripped = [w.translate(table) for w in words]\n",
        "text2 = str.join(' ',stripped)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zP6_lhEM6pz2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N5uT8VEg6rdm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(text2)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}